package com.itheima.spider.tencent;

import com.google.gson.Gson;
import com.itheima.spider.dao.NewsDao;
import com.itheima.spider.pojo.News;
import com.itheima.spider.utils.HttpClientUtils;
import com.itheima.spider.utils.IdWorker;
import com.itheima.spider.utils.JedisUtils;
import redis.clients.jedis.Jedis;

import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class TencentNewsSpider {

    private static IdWorker idWorker = new IdWorker(3, 2);


    public static void main(String[] args) throws IOException, SQLException {
        //确定首页url
        String topIndexUrl = "https://pacaio.match.qq.com/irs/rcd?cid=137&token=d0f13d594edfc180f5bf6b845456f3ea&id=&ext=ent&num=60";
        String noTopIdexUrl = "https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=ent&page=0";
        //执行分页
        page(topIndexUrl, noTopIdexUrl);

    }

    private static void page(String topIndexUrl, String noTopIdexUrl) throws IOException, SQLException {
        //热点数据获取
        //发送请求获取数据,获取的是Json数据
        String topJson = HttpClientUtils.doGet(topIndexUrl);
        //  System.out.println(topJson);
        //解析新闻列表页的数据
        List<News> topNewsList = parseNewsJson(topJson);
        //存储数据
        saveNews(topNewsList);
        //非热点数据获取

        int page = 2;
        while (true) {
            //发送请求获取数据
            String noTopJson = HttpClientUtils.doGet(noTopIdexUrl);
            //解析数据
            List<News> noTopNewsList = parseNewsJson(noTopJson);
            if (noTopNewsList == null || noTopNewsList.size() == 0) {
                break;
            }

            //保存数据

            saveNews(noTopNewsList);
            //获取下一页的url
            noTopIdexUrl = "https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=ent&page=" + page;
            page++;

        }
    }
    //保存数据到数据库
    private static void saveNews(List<News> NewsList) throws SQLException {
        NewsDao newsDao = new NewsDao();
        Jedis jedis = JedisUtils.getJedis();
        for (News news : NewsList) {
            Boolean flag = jedis.sismember("bigData:spider:tencent:docurl", news.getDocurl());
            if (flag) {
                continue;
            }
            newsDao.saveNews(news);
            System.out.println(news);
            jedis = JedisUtils.getJedis();
            jedis.sadd("bigData:spider:tencent:docurl", news.getDocurl());
            jedis.close();

        }


    }

    //解析封装数据
    private static List<News> parseNewsJson(String Json) {

        //将Json转为java对象
        Gson gson = new Gson();
        Map<String, Object> jsonMap = gson.fromJson(Json, Map.class);
        //获取本次一共有多少条数据
        Double datanum = (Double) jsonMap.get("datanum");
        int datenum = datanum.intValue();
        if (datanum == 0) {

            return null;

        }
        //获取map中的新闻数据
        List<Map<String, Object>> newsList = (List<Map<String, Object>>) jsonMap.get("data");

        //新建个list 封账news对象
        List<News> listNews = new ArrayList<News>();
        //遍历集合获取新闻数据
        for (Map<String, Object> newsMap : newsList) {
            String docUrl = (String) newsMap.get("vurl");
            if (docUrl.contains("video")) {
                continue;
            }
            Jedis jedis = JedisUtils.getJedis();
            Boolean flag = jedis.sismember("bigData:spider:tencent:docurl", docUrl);
            jedis.close();

            if (flag) {
                continue;
            }
            News news = new News();

            news.setId(idWorker.nextId() + "");
            String title = (String) newsMap.get("title");
            news.setTitle(title);
            String time = (String) newsMap.get("update_time");
            news.setTime(time);

            String source = (String) newsMap.get("source");
            news.setSource(source);

            String content = (String) newsMap.get("intro");
            news.setContent(content);

            news.setEditor(source);


            news.setDocurl(docUrl);

            listNews.add(news);


        }

        return listNews;
    }

}
